
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "54"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = os.environ["ANTENNA"]
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "222"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, '*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 124 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 119 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H5C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H5C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        antnum = row['Ant']
        if antnum != int(antenna):
            continue
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


html_to_display = ''
for am_html in auto_metric_htmls:
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line: 
            break


HTML(html_to_display)

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459594	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.350352	-0.209439	16.988489	16.762608	9.268155	12.274014	12.366482	14.045535	0.7524	0.7410	0.2872	9.233131	7.873946
2459593	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.956886	4.233521	21.967115	20.430964	14.312862	18.729460	6.341118	4.467689	0.6950	0.6917	0.2715	5.657257	5.173156
2459592	RF_ok	100.00%	100.00%	100.00%	0.00%	-	-	0.202751	-0.183178	18.923793	21.384136	49.823725	7.162391	-1.621015	-0.079987	0.0362	0.0315	0.0011	nan	nan
2459591	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.093287	5.886476	24.375888	24.398738	10.850839	15.827821	-1.556481	-1.926328	0.6387	0.6476	0.3786	6.301953	5.308350
2459590	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.034935	6.183421	23.023115	21.525900	12.813276	18.545787	-1.287791	-1.755635	0.6391	0.6462	0.3741	6.165606	5.635948
2459589	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.648015	5.314454	19.863540	18.611068	8.452723	11.273432	9.988995	8.346040	0.6849	0.6737	0.2945	5.959070	5.480492
2459588	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.498633	2.768930	24.597096	24.945193	21.976878	11.912354	7.215422	7.528746	0.7410	0.7152	0.2566	9.512709	6.956280
2459587	RF_ok	100.00%	-	-	-	100.00%	0.00%	4.178995	6.416683	28.068721	34.928460	-1.599171	-1.707873	-4.675000	-4.393628	nan	nan	nan	0.000000	0.000000
2459586	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.157286	1.941862	1.417748	2.422907	18.799204	6.152386	0.725578	-1.433000	0.6324	0.6423	0.3652	4.412158	3.917334
2459585	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	11.846857	6.888980	3.009964	4.081852	8.220783	11.074551	6.429654	-0.094023	0.0383	0.0333	0.0015	0.000000	0.000000
2459584	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	17.257556	22.285822	24.070031	36.571947	5.855813	9.708957	4.546019	-1.344049	0.6000	0.6077	0.3807	0.000000	0.000000
2459583	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.137628	5.751849	6.442756	10.410168	27.576544	12.383479	5.441290	-1.612666	0.6432	0.6487	0.3913	5.362251	4.852830
2459582	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.140031	4.979679	6.089671	9.219758	44.716264	15.532638	1.626554	-1.627647	0.6430	0.6503	0.3937	4.415916	3.658511
2459581	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.607215	7.601591	9.583297	10.511374	10.809526	12.994753	-0.018606	-1.888546	0.6466	0.6545	0.3917	3.694329	3.388043
2459580	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.549700	5.632722	7.794236	10.883757	6.556383	13.121177	3.145610	-1.819566	0.6491	0.6525	0.3898	7.586789	6.032908
2459579	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.190633	5.605686	9.841773	12.387551	37.167410	15.381661	3.736370	-2.184420	0.6342	0.6428	0.3974	8.804438	6.017123
2459578	RF_ok	100.00%	100.00%	100.00%	0.00%	-	-	-0.861819	1.438281	9.827889	12.829803	35.284090	-0.588728	5.741296	0.416007	0.0423	0.0330	0.0020	nan	nan
2459577	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.610300	7.135637	9.331304	9.653764	4.184896	13.531352	-0.901923	-1.812116	0.6401	0.6501	0.3934	4.710021	3.920915
2459576	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.605152	5.251349	7.330733	6.972046	10.272531	11.798416	-1.131495	-1.728568	0.6641	0.6740	0.3913	5.691836	4.576691
2459575	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.308504	4.400325	7.431659	7.476990	4.923989	9.548510	5.694425	4.574479	0.7239	0.7266	0.3479	4.182840	3.977399
2459574	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.464009	5.256727	7.118478	6.829405	10.479142	12.798606	-0.454241	-1.131783	0.6584	0.6681	0.3944	3.809502	3.404597
2459573	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.948788	5.598665	13.875733	11.954761	11.612495	49.454204	-1.143178	4.911372	0.6569	0.6664	0.4002	2.957017	3.001661
2459572	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.705899	5.491479	7.303821	6.989453	7.925364	9.195039	-2.348768	-2.697458	0.6659	0.6762	0.3933	4.878982	4.019481
2459571	RF_ok	100.00%	100.00%	100.00%	0.00%	-	-	1.120166	1.287693	12.978204	12.898367	1.336916	-1.775903	0.533378	-0.773930	0.0433	0.0324	0.0015	nan	nan
2459570	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459569	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.525834	4.377953	8.479728	8.510881	6.844006	6.504742	5.650789	6.211237	0.7459	0.7484	0.2420	6.609461	4.932049
2459566	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.865690	5.167483	9.359838	9.142284	8.728004	25.961249	0.565037	-1.417029	0.6602	0.6687	0.4046	4.655047	3.951100
2459565	RF_ok	100.00%	-	-	-	100.00%	0.00%	6.613916	6.613916	1909065.330398	1909065.330398	-1.379264	-1.379264	-1.415126	-1.415126	nan	nan	nan	0.000000	0.000000
2459564	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.573519	5.328413	5.271174	5.298543	1.138458	1.244962	-1.488220	-2.047291	0.6548	0.6541	0.4007	2.890068	2.445667
2459563	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	8.415518	9.342660	28.677185	28.385035	15.779296	21.686618	-1.538465	-2.246831	0.6533	0.6567	0.3940	3.979976	3.236357
2459562	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.087661	3.330326	2.195784	1.866980	1.130808	6.655264	-1.118000	-1.329709	0.6574	0.6651	0.3910	3.085709	2.574678
2459561	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459560	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.986529	3.064286	2.345573	1.842561	1.222841	5.301270	-1.369305	-1.525350	0.6521	0.6643	0.4031	3.390273	2.841573
2459559	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.914456	4.397369	2.981141	2.809770	1.540350	1.082148	-1.437129	-1.829699	0.6500	0.6610	0.3945	2.953815	2.491237
2459558	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	8.465776	9.832201	15.523948	15.292897	17.263715	19.508266	-1.895058	-2.573886	0.6517	0.6495	0.4000	6.977140	5.518962
2459556	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459554	RF_ok	-	0.00%	0.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.6589	0.6588	0.4008	nan	nan
2459553	RF_ok	-	0.00%	0.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.6549	0.6564	0.4016	nan	nan
2459552	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	10.376924	11.418461	21.303467	21.617947	24.222487	27.257027	0.244557	-0.384473	0.6595	0.6584	0.4034	0.000000	0.000000
2459551	RF_ok	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.154218	3.553806	2.942730	2.704629	1.601367	1.561274	-1.713673	-2.403673	0.6583	0.6617	0.3968	3.265400	2.603751
2459550	RF_ok	-	61.29%	61.29%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.0341	0.0313	0.0011	nan	nan

Antenna Season Report Notebook¶

Antenna 222 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 222, Node 18:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 222: 2459650

Antenna 222: 2459649

Antenna 222: 2459648

Antenna 222: 2459647

Antenna 222: 2459646

Antenna 222: 2459645

Antenna 222: 2459642

Antenna 222: 2459640

Antenna 222: 2459639

Antenna 222: 2459638

Antenna 222: 2459637

Antenna 222: 2459636

Antenna 222: 2459635

Antenna 222: 2459634

Antenna 222: 2459633

Antenna 222: 2459632

Antenna 222: 2459631

Antenna 222: 2459630

Antenna 222: 2459629

Antenna 222: 2459628

Antenna 222: 2459627

Antenna 222: 2459626

Antenna 222: 2459625

Antenna 222: 2459624

Antenna 222: 2459623

Antenna 222: 2459622

Antenna 222: 2459621

Antenna 222: 2459620

Antenna 222: 2459619

Antenna 222: 2459618

Antenna 222: 2459617

Antenna 222: 2459616

Antenna 222: 2459615

Antenna 222: 2459614

Antenna 222: 2459613

Antenna 222: 2459612

Antenna 222: 2459611

Antenna 222: 2459610

Antenna 222: 2459609

Antenna 222: 2459608

Antenna 222: 2459607

Antenna 222: 2459605

Antenna 222: 2459604

Antenna 222: 2459603

Antenna 222: 2459602

Antenna 222: 2459598

Antenna 222: 2459597

Antenna 222: 2459596

Antenna 222: 2459595

Antenna 222: 2459594

Antenna 222: 2459593

Antenna 222: 2459592

Antenna 222: 2459591

Antenna 222: 2459590

Antenna 222: 2459589

Antenna 222: 2459588

Antenna 222: 2459587

Antenna 222: 2459586

Antenna 222: 2459585

Antenna 222: 2459584

Antenna 222: 2459583

Antenna 222: 2459582

Antenna 222: 2459581

Antenna 222: 2459580

Antenna 222: 2459579

Antenna 222: 2459578

Antenna 222: 2459577

Antenna 222: 2459576

Antenna 222: 2459575

Antenna 222: 2459574

Antenna 222: 2459573

Antenna 222: 2459572

Antenna 222: 2459571

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶